{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>851104</td>\n",
       "      <td>2017-01-21 22:11:48.556739</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-01-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>804228</td>\n",
       "      <td>2017-01-12 08:01:45.159739</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-01-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>661590</td>\n",
       "      <td>2017-01-11 16:55:06.154213</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-01-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>853541</td>\n",
       "      <td>2017-01-08 18:28:03.143765</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-01-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>864975</td>\n",
       "      <td>2017-01-21 01:52:26.210827</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-01-21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id                   timestamp      group landing_page  converted  \\\n",
       "0   851104  2017-01-21 22:11:48.556739    control     old_page          0   \n",
       "1   804228  2017-01-12 08:01:45.159739    control     old_page          0   \n",
       "2   661590  2017-01-11 16:55:06.154213  treatment     new_page          0   \n",
       "3   853541  2017-01-08 18:28:03.143765  treatment     new_page          0   \n",
       "4   864975  2017-01-21 01:52:26.210827    control     old_page          1   \n",
       "\n",
       "         date  \n",
       "0  2017-01-21  \n",
       "1  2017-01-12  \n",
       "2  2017-01-11  \n",
       "3  2017-01-08  \n",
       "4  2017-01-21  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('ab_data.csv')\n",
    "data[\"date\"] = data.timestamp.str[:10]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "plt.rcParams['font.family']='SimHei'\n",
    "plt.rcParams['axes.unicode_minus']= False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. 检验指标确定（10分）\n",
    "一类指标  人均停留时长，保证一类指标不下降或者在正常的波动范围内 \n",
    "\n",
    "二类指标  广告点击率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.确定检验统计量 （5分）\n",
    "一类指标  人均停留时长；我们用 人均停留时长 之差作为统计量\n",
    "\n",
    "二类指标  用 广告点击率 = 点击用户数 / 曝光用户数 ，我们用两组 广告点击率之差作为统计量\n"
   ]
  },
  {
   "attachments": {
    "image-2.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAesAAAD4CAYAAADMxs4gAAAgAElEQVR4Ae1d27GzPJB0KpsGAWwoJLCJ8PolQiT/G8Gw1ZJGmhGSwD7GRqZP1SkL3ekR0xpdH//+/Vv5TwzYBtgG2AbYBtgGrtsGHhAO/4hArwj8999/vVa9i3oT33PFRHzPxfdXckc7IVn/ijRv+h5UducKnvgS33MRYO5HECBZH0FpJ84yDetjnHdi1YLndXyM67zUwnP/ZZ2GYZ2y+PP4WMfjmeSZdv1MMjlXfMSX+J6LAHM/gsB9yXoe18cwrZ7zlnUah3UYyv9jzowZsiDrXa5epnWs5P94PNZHMWxcN12AZVoHqfcyr9O0uHc4VIes3r/ySDI5Ikl08tChOxLXxrkLvm/pdFvoDj19DV+nS54xFA69zkmR5nUa56Cv13Vd5nUc9uq+rPM8+/8p6N9xXueirh83BtD+i5QNp5SuHY66ZTZXSlpw3Zes0VCj5gKoiRghTOFnfMCDPBQAhJf7yB8loj/SAGzZlSKit6kPOhzhHeAfXyfGvofja8ruk/A2OnupHesKLes0WWbW7RoxXVvSClAnV+5b4BvweP0bAnlI51+Bd8D5PL4oq6Rvcr99/bPMILGH04V14mgbM2UjJ+jTVrutvkOh3krXCaQL/Jqjkp7QJ5Dikr+dlVf+bazzmBlvhTqte7q7TtYo77Hb2ZA39b8ka4eDBV0LzpCjxS4+PUWUhca7taxLDQPFhSHzUDLKfUgnAda5uIdh3RsNiJX/AceessMUgcPYYQTF9JpS/RxUs5vqiEoQDILO5eORKRAo50fssNn6Ld6CUGSs27VXRsdw+D18LVLyZL4nQyS171FS4tcqfx2y597Dd5u+UBbIJTMqjLy3mSgfb4HmdKYiVJxWH1Uilb1de04GUjmS+AbScx0L2yHBdx2/EyczLSuN07IavjbG2uq+FQOfwdPyg9RqfYmst99lyq/tIlk7fCCMNBRtyBONwUhxC+izZG0toW1DqH1knnSkgdsP5ak6bF+ha589Zefn8+UVl3VGJ+dFK0hyOfcXlkwgUlEq8psXXPMP8dCW0N6giGO7Qho1kpRnmT//Hr75G/rn499QydL0HUJLHCCW/Q7RHr7b2moSCqGGXLxflHeWwTLlVqMlQP8OomeyxPqxYO3q4JYbWO/p1Zj+5XI0TnCrd8rw2mBlwrc62tet5i81R3gapfVD90+8t2QTfu9H1hCC64Glj2ucYMkkQWrBlRsVBK8aeGbVuvxr42lQlFlv0HQOBm8tb/oHrt5jbHColwyBQ5bHFU3WAn7gcU/ZWbLGC9uOzvUgeB9Z491kCDC26yWzMnYA+D18yy/8t29IE0M5/5rvHr7bdIWyDLn4FFHeWQb771nIP+RhiL6k9wZt2aaCURfdkfEjXdavrDfxrdrRozyvmO9G54b3cPPbwzqOYdpQ5rtj9RAv6X/nbfD0pDwVOjlJd5feW8gaBsK4Dg9F3LHs4477kbVgYywS20PSjRwNe9sDrDdml73JWwqs/dqyi7EcwUPQKu6CeZgUe/8DTHF/zbWn7HbJ2n28aajcDpML5pB5iGMU0l440PZDX15BYQRHL4xBesg25O8ssQpZZ508r6SsIvPDsQUl6DqBafQoKrgwdGhnt20L+T185f2AucKqRD4bApC0+e+OTsijq+c9fFXU4PQEZmRYqDumxTad/kMd+/q7bPSMVkKNYeGkU+V7Ca+idaV2y5uOmLYa1foiNUKkgSmkjVNH45R224RpSMHFj1YWpg0LZL39RrJ30fVxbvm2Afqc6rCJd8yDZB1BTYos9ZYGbwWLZCOm9cbss9OL12Iip7RLq85NeaI8NmUiH2kchWG4wsd6l3nrPWVnyTrMGUV88ay3vYVh8qikPebolcvOOCisNIy+F+6VyzCllZ9Y1JOGof0HDYUk+Xs5y1ycJWOvLNFLC21Bddh0S9NupJEO50bZ6ogV9+/hKy/6wndcWHMSO00Pb72hrZh/07mTstPvHr4pprgK9Tbk4uMlgpR0/ne/DRTyD1nYtGiDeodBaJO2OPeU6pLF0SSr3Y7fwneS+TuCFT0Zf+13IlVY0kclXsd+3eK11JFzC9lCnaLqkG9Q55i1j6Juj3WW/DOrXuen3CRrBYY4U8MSn/y30LM1Aig3nDwX/5w1XngWPrxq3JCp/YiC501+9pSd7z1ryzkRZxEiKIc415grJHyxmMqQD2wvvKz4UgfCpxcy9fXJLespbS1Epyy0NSgCTxSlIbj0Zmgbkv8r7eT38BVsyrKRUCfn2GmLvhXHTl6VVPDew3ebtFBWQWfU9BjagF6Maix017bQxspz7bb9oO1q6x3P8l3YWqe6+PYubdgskMxIOeZQ848RGg7gYnSzEGT2m8vZ4KneCyQesVH+xSogPOGTMChG3vUkWRcg+iuohSydgjdDbqoBbXpfykq2bajeOOxHVKzBz3ruKbtEjB4Ca9m67rIbDnVycNhrZWU/OJ+DnvPeCXfEnjoKugxPoKX0GVljdb9Ya2g3wQ0llPyn7Z78IHH9/q+0k9/DVz6FP3S6N4uetgSqcZcSS797+G7TFOqtdIaQ05+GwWNn1JZu20/edvF8hKxVHE3E2q2LFf/MapX3zH/NiKIhXWS6lVOxU2bS6feCW0YTtL+usLgRnsj6mR0YkoP+JVlDV2eNYEOeljG9ZeVW2GIIVVbtzusobjdHKQLVcJfcBYGbhqLTFOKGYPsR6TS/795TdiWlCbw8WXrFhw88jihDOZjes/rgHJxPknXMqySL7IN2UTKyDu0v1jkoL7RbN5weK17KX9f1tYWIv4dvCacn/Tbf6JYEjnb69/Dd1My0zxC6qU9lbtcN3B3VTZuSs4Wstm3FqZltsrQTwQ0dp9Ghpyzr0O4L2SevHIdsONuVF0ekxLoujISafDK9O49ha2zmn2oRXNtvG21CRrk20Xc8bkrWy+pWNYbeaH5MZ/sjgwCksVthIF3kdTSsTe+0MNdcWviDxpQmRpQI642DZK1gypxNsq4oPjvU9Qey3l15btuQr7pvY95igCKZwp5pzIVO64w5b3QWZd8p9ter/dT69dEu/rprYI9M+sM3IBSUP7ppT3e6jTJHfh8k641VX546K+uxnGB1a9l3az3jOo9O4UlHt9SWfZ6pLpkO0wSs3boq4h9ObEReuTXtnmEszdM66XnqA3J6zrI2FauOJPhYGR7YheE6K+CJZg9bFxLd9yPr0NPCdi2zkDFCUu+RIkpJ+UVizT4iKLEYpvK3zqzx+kJsg4sJCnFDmP6IYvSbOJ4lEzcMLitloQj0KUhmVTYAzD44h6lWeHvhaE++5x4/T7N1pJS+YFmHD90pI5x+hI7mIEPfONQi5p6kDkUl7xl8S+0kKdKUVLt+D1+8HXB/tdNdIsfPkXWpc1Ra51KSayJYLeHjbiFDnB4WF0m6BZP+sKFiO3RwSYcXuL84DF6oZqk9m2h/IGs9ry8LzEzerg2pd7GBSneErVtxF0hY5KpXqW/Sbj3uR9Y5BtkQOHpom2FwN0+IU6TwQWaLeRbdk8uUJvLOjnzUxTslDis6Dp/r0JI7a+gqSvEDVuG/7DxCJnquGFundO87jbKAAEdvucah6xKZPkfWIAYozlgH04ZK+S8rVrFq+nV1dB/3Eqdh0B7d7oLcql6kPFGQSfpb5YZ32cZLKfYXQEk7Nu+nOg/Xw/cNnW50zAe96hs7RzKLbwdXwXiv/Uo89yudS+NZ6jwUjA5NsHn6A8/Yrz+7M7bLOgtE7o8uVVulQr6ujYR1Ola/+vUcaRRpu0GqaPmGr2PbnrMXqcgprvVw6z/s6JPLwZB84Rt1ZxVoPZCV6x6RDu+HrWHbRa2+wxN0jv7YS1mFhYi8IrMCDr37QOApZdfHK1VrCeUEZdcasXHDhLVeu1P2XoHgcB6vSNqa4ufwfUenG8TlFDYOmCkfMlOybEuCPY6v74TFqbZSZvBzo4cZobp3bnfKatk5f5c+WM/t5rKChHIb5RAWMG5KL4c2GzvPoZbhHd0iugK/x3cxpBt9rQP55+Xu7Iv2HdTC/myTszfuVL/VhMpDbYRXwuUX7YRkLWjwt0sEjiu7Ll/v65UmvueKgPiei++v5B7JGg7+EwO2AbYBtgG2AbaBa7YBWtYX73rhw+FfHQHiU8eGIUSACPwGAtBzJOuLy5Jk1BYQ8Wnjw1AiQAT6R4Bk3YEMSUZtIRGfNj4MJQJEoH8ESNYdyPAcMgrbDnZWdiZ4CtsX3MJT7FU9nEnK7o2uc/B5YwWZFREgAkTgjwiQrP8I4CeS/4mMCvvI5fQfu99R7xEtbPTX2yfCSUKg6N19jh8A6E/4fKB+LOIEBMIe87S3+7VTodo1k32y2Co3rePmRMJ26tdD9/bvug9vu53p9QKZsgMESNadCOn91YQiKpBypSCQctzbiz2OYV9ii6xNmmCFR+UqFw/kB3pUym95k6xb6PxiGMgMBKoOmsA+4IecSPamd3YHapRvn3pTCa9nozvPr+fClB0hQLLuQFhvIaOChb21rLPT2SI2tqcPEo5H8QnphtOJ9I03JbK2Zw/ISVvHOw2xSsrxFnxUfnReGwEcSBE7jrqq4XCLt03KqE6pLuYSbpL1JcTwyUqQrD+J9otlvYWMgiJLVdha1rVThvxpPUKoW+K2BKxK0NZ4nN9O4eJy5F/LRCI1ft+CTyN/Bl0JAdv+WjUzx5y6Y32VJR7PdcYpU/7IS3cMbWB63+aDvzs2VJcr6zdCWne6lvj5y3p8Rzh0fsPpX+LXOnDLv48uCz7SqVX1dOdxX9TqbwmFYS8jQLJ+GbrPJXwLGYGsobCCBVw6A90d3ZebJeE8YZyJDiWTEyueazy7b1kLhrlyEv9jv2/B51hRjPVtBA5alK6d4qKT2J5BePoqRN9ZxRnREkfSxCTGstZt1BMz7hWXtHLhi/Zz+Y3TOsWz/wPp1j6YiK0uq3QRjP+WcZFLrGtMS8evIkCy7kCy55CRV1bNXr4jeJwnrOIu9rYyKKSa7kGYHq6EsizH9cqveCvoAfmcg8+Bghnl8wgcImtLdqmS8JfzsdHmsvbo2ruMIIUztmOD1Xn6tLptyy1e1k+XF2rxdP11uelNNh2LFETXjyJAsu5AsH8no4P3aMPqLjKmkHUhn2zOGha7zFuTrDtoXL1V8QjZNeKkDmOpg5gR445lbT+VA/kB60bdkihUPWrxa/4pE7p+DAGSdQcC/TtZl15SCFiFVW+oKcQNyWhZK/zo/AACBWs1L7VBZCTrHCw+94IAyboDSf2JrAurwGXeerMaXFnJcfTP4XMyWUO5/mEP65/w6UD+rKJFAIRbHAFCO3JbAZVlapJqoj9gCV/Bsl7L78JhcCPYWzyQrDsQ8zlkVCDgL1nWVeV7UDbn4HOwcEb7AgIgsPY+ayEzvQAM7UzOB5AFYXYoOyPGS5D1doHZgpXg6FhzgdkX2t73iiRZfw/7wyX/nYwKc80D9krjg1cnlzkFWFpfWiD2UPu/DYOH1bFPHM5SAu3v+JRypd+lEdicYDauU2JmV3W7dQvrMfKtW7LYTN70mmTNrVsin3v/kqw7kP85ZFQg4GXaKDwPTyFuwO1ZsnYdBDfcjj2jQxi2/JsQzsHnb3ViaiLQRMB1NsYVW8fyf2vtN3Nh4I0QIFl3IOx3k5EbDnSHRBzdp1kna+Rl57c/D+i78fn8G7BEIkAEiEAbAZJ1G59LhJKM2mIgPm18GEoEiED/CJCsO5AhhMR/YsA2wDbANnDvNvD49+9fB5R13yriA+VfHQHiU8eGIUSACPwGAtBzJOuLy5Jk1BYQ8Wnjw1AiQAT6R4Bk3YEMSUZtIRGfNj4MJQJEoH8ESNYdyJBk1BYS8Wnjw1AiQAT6R4Bk3YEMSUZtIRGfNj53DJXtiXFfPw7eKZ3383FwsoNXSuU3zjYvRaffPRAgWXcgZ5JRW0jEp43PHUPz/f8Ljg79w/nzH8WQZP1RuHspjGTdgaRIRm0hEZ82PncMzckaGJT8LokNyfqSYvl2pUjW35bAgfJJRm2QiE8bnzuGlojZ+C3zOg448tb/D2N+ml84t16Oxh1w9vi0DubyjG2c/aH2fBh8m8e2nDtKkO+cI0CyzhG54DPJqC0U4tPG546hhpgdADgyV47GBUHCLZPYyzpPg7qRa3vT1QpydxffJFKfR3s5iLsNa/dSGkvWqAduApOarLCqs3LuKD++8xYBkvUWk8v5kIzaIiE+bXzuGJqTtVyZGUkxB8UMPVtCjVEx7x0ta1jmibglTl6u+Kdfnbd2pxi7dU1R6boRAiTrDoRNMmoLifi08bljqLNYZQgbl9YMk10N7obBwzWxEk/I1xC3Qk/7iwUsadXv0Lw2SxG0zk8V46xrqYv2p/vWCJCsOxA/yagtJOLTxueOoW0LF4T5WMdpscPPQpBHSLQWZxdskvUuRIxQRIBkXYTlWp4ko7Y8iE8bnzuGNsm6RLT5EHdpm9eROLtgK7JetTsl5DB4woKuhADJOmFxWRfJqC0a4tPG546hu2QNMpYJbFkZLpZ12OaVL/wax9GsBndD7WZxGOaxx3VuAm4JOs/DLVLDkLqqSzM7Bt4GAZJ1B6ImGbWFRHza+NwxtEnW67ou0+hXXTtiHNe5sC1rGtOc9gBS3ljkhW1X0gGogm7Jel0LeWzqUs2MATdCgGTdgbBJRm0hEZ82Pgx9AwLLrLZ+7eSHuOO4whLP/5trz3ayZfC9ESBZdyB/klFbSMSnjQ9Dn0cAx5MO6tCUx2NwC9Kez4kpiMB7ECBZvwfHU3MhGbXhJT5tfBhKBIhA/wiQrDuQIcmoLSTgw39iwDbANvDrbeDx79+/tjZk6FcRQAPkXx0B4lPHhiFEgAj8BgLQcyTri8uSZNQWEPFp48NQIkAE+keAZN2BDF8jo7BFZHcriQCAiw6GNV+t6rfA7GSCfapqvykuOBjbm02l0Lf8vobPW4pmJkSACBCBjyBAsv4IzH8rpElGy7SOw7AOhX93/V/Bfygd3KD3kGLrSTiKEacp7RGvO3FJRwo3FGmvLQLYX7rd2pJvdfHP2wsTdH5NfHREuokAESACnSJAsu5AcK+RESzlvdOU0suDcOMFBDhWMTDtPlmXLXJ/1Z86JSoVFV0gazHAcUhFtOqxbSY9FG83ipmsq1tcpp/pJgLuZDB9uYa5rzo/mOQveFXafzPLA+XrznMzLwbeBQGSdQeSPkTWBQt7a1krQjTvbZWHs5QfwVp3Ci9Z7rj8wPzJecmV8q3Fn8gZebj7gMW6xgiAcj8GsbpRNi1rgzkfdhGwJ5iF+6pjO7LtfTezZoRXyLqZoQ8kWR8A6V5RSNYdyPsoWeNIxPS3taxBjjnXIr63QoRIrSJrW9Yo443nGL+ooA7hk4Ch6wYIWLJ2rXwd4+Ucto3/DQ6S9d/wY+qjCJCsjyL1xXiHyAhE5+7tTVZwblnjFKYNWbuTmsZ4AYGzqhXpt8jax0XaZPkuy5yuHXSY+bOPx3hrggfSnc0slrT8DsOaLGqxrP3vMOmOiBXGIXxsEj79OALPkrU5KxzfkVow6aFaVntWuLT5LVm7EaPNh6YBzzsLPB9co0N3GQGSdRmXS/m+RkZby3rzUo7gQeAqLshWjXRXyRqXDThLBTcNieLyVrrietjt61jqJGwq44fFbdpCpILXa/gUMqLXzyBgydqToVkHEa1sXOqBTuKUbuFyl2s84roNgIL80GH0nwbywzOeLFkjr1RODU5L1sgrv+ELHW/evFXD757+JOsO5L5PRr7Xb+eHw41B+WrwYo9fyLqQTzZnjTLsvLUl61XmsAXXnaHteUoW9IB5crGy468oSMlw+7uPzzYNfX4bAUeAZoGZbkeaLLVbY6I7mbU4iJ/I+hhRI43OT7tT+dKBUP3mFEjXLREgWXcg9tfISAhYvaBZZa38ncKROWvt762OtrWbkbVRRM4kMRaKzd1b08X+g4uY552n9s+v4VPOi76/gQDIWrdbd0903B2hCLLRmYx5NOIIWY/T6KZw6pM1GtcD5TfL1HnRfRcESNYdSLpJRoVV2GJh53PWbmgtrPLWiswrnHeRtR0Kh4Vgy7KAQyFurWmxtvdXgiO3Jj62OD7dBIFItOp9k+V7gCzD0Ldru03iRKfYD2PPh4bAUaED5TfLVC9F520QIFl3IOrXyOhblrW2ptMQYQ3m2gp1H5+WdQ03+rcROEzWmjhNliBUWZCpyNXEwYNu497d6pz65Do/7U6Zcxg8YUGXR4Bk3UFL2CfrwlwzVla7RSppdbhbLV4ccy4Qe8BlzzJ2VoJaYOaTLX6R2gHrgGTdQQPssIo5Wbth8Ar5CjGmDQt+AZkcDITXdyNABxaYucOA4nB7DThL0MhbLzDzdeUCsxp6d/UnWXcg+X2yLr1EgYCXaZ2SRlKJCnFD6FGyxkIxGX6Pv2GBT3yOi9Nmd7oajhIdhp1h8MewuiNHsVpX1Vg7X8NH50D3ryHgCFAtMMOWwNT2LVni3e3WLazo1gvSEAOjPA/fATZbu7w1rfvAjvyb5nVePrdu/Vr7O+N9SNZnoPrmPJ8lI1FUgzlisVWpOlkjr6becUosbd1qlXJW2LP4nFUP5ksEIgI4Xz/uaJA1GP5XE3uMTwcR2EGAZL0D0BWCSUZtKRCfNj4MJQJEoH8ESNYdyJBk1BYS8Wnjw1AiQAT6R4Bk3YEMSUZtIQEf/hMDtgG2gV9vA49///61tSFDv4oAGiD/6ggQnzo2DCECROA3EICeI1lfXJYko7aAiE8bH4YSASLQPwIk6w5kSDJqC4n4tPFhKBEgAv0jQLLuQIYko7aQiE8bH4YSASLQPwIk6w5kSDJqC4n4tPG5deiSH2byrTMB8oNQClI5cOJfIRW9boIAyboDQZOM2kIiPm187huKw34e6krXZcVlG/oY0T9jU73J7oWcSdYvgHafJCTrDmRNMmoLifi08blvaMmaDaf1vemiaBwtOrzrSDKS9X2b6oE3J1kfAOnbUUhGbQkQnzY+9w0tkXWOxvZcbnN8fmMYXY71dRfmPA4cy/vQ19Buy53maR02l+Lk9eXzXREgWXcgeZJRW0jEp43PnUP9jVr6Eg+LBm5905d2uBuv4q1Z/vatMbL3dhj9uGVtOw6O6Ed1WQisandL3rfm1C0ufLoeAiTr68lkUyOS0QYS40F8DBx8yBFwl2oMjgxxuU3k3solNM3La7Kh6tfI2hK3VNd3LEjWggd/LQIka4vHJZ9IRm2xEJ82PgxNCBjLWaxZfZWmXOsq89BuGDzcDS/x1FD1S2SdEX6sXc0/RqDjzgiQrDuQPsmoLSTi08aHoRaBaDnvkiMsYL+aPK5Hy9KQrC22fDoPAZL1edi+LWeSURtK4tPG57ah81jcphXJGsPgZtFXhlRGzC4Uef7Vsq6Uy2HwDH8+GgRI1gaOaz6QjNpyIT5tfO4bGizjNEm9LiBbRdDbhV44RCWs2gZZI66Y1bIyPCPrY/u2bccgL9cNz3OB2X2b6oE3J1kfAOnbUUhGbQkQnzY+tw4Vgo3zzeMq09Eel8IWKiHndV2XafSrtB2Rjuucb69yhP5YsX2rvd/akvW6FsrN87614PjyOQIk6xyRCz6TjNpCIT5tfBj6IQTcqvNxHcftv+0gfKg+LOanECBZdyBOklFbSMSnjQ9DiQAR6B8BknUHMiQZtYVEfNr4MJQIEIH+ESBZdyBDCIn/xIBtgG2AbeDebeDx79+/DijrvlXEB8q/OgLEp44NQ4gAEfgNBKDnSNYXlyXJqC0g4tPGh6FEgAj0j8DzZL2ow+fd5v4hboVY5jntSdxgg6vphnWcNwEf9XAHD3yoErgkoFhUdrDCHgAkozZCxKeND0OJABHoH4GnyRqHCgxyaIA5CB/7CId1iicIbMEBeaUtDDh8YFiHzb++Ri7kYchtWSfclLNJ5/3GVMC2Am7fZIVAS7Hdu9bL0nUokbJ9X99ZmdBZwfuUEpTqsK5uvroS1PDO93U2onYeRLLuXICsPhEgArsIPE3WyDGdhwvC9bfEJL96mZa8UtqUIlwMnzy8CwcPRHKzcXSeR+rgLOtHiYDzwxI8qbYPOvDV03XQVbf+6n0/RdbqpCVdL+N2RyqmU5pQZ90JKbm9KNqdplK62MlbpkpHrVV2QT7hRUjWRqJ8OIiAO0VMDkuR3yPfzMH8GY0IvBOBw2TtTvLZWLPh5J7B/z5UuLZwhQDcJe0uDqxnRV7rsvpDgywRxxd9M1lH3o8FVBwg1SKxW1LBe5k8YZFP/oSiCScgwdoPJx05EguKQQhNY1WqyS4ZVcjPnaqkZOLLK4xcOMLGiEOoszrByY0CPKXA/mDRO4wK9SuBovx28VFx6SQCgkA6I1x8+EsErovAE2T95PBxYTi6Zmkmi/goWT9W6RikDkC4r7ZQroYfZRli1YG5O5DuuszrPM/rsizuH24cOziNmBLwJKeTShn+fX2nZNblnmRZY81A4lndGcIoQWs9AWq/uPUGVkYYsX8CL2Tz5LvluB0ZydBp4CZZ54jw+QgCJOsjKDHOVRB4gqzHQHIgAWtZioUow5w4lL40dz2P2mpKZFIl6zhnLBb8sI7TvE5xztyTifBzykfDm9XXWbVZ/WvsLWQNInME7UnbL5TzxJ3IMZUphCe/CyxfdXnAs4R2lIyq6wmc5Vy61H5Zl2yNga+zt7CH0ohJYYGgGXUp4TuUh7BRVmo7cmew9XPhNfkEyI/ikyREFxGA7shGxCIoMBqwvga6A4ZB+nbMWeE4D3zUHWRJ56eIvCER2r7khe9D6a9YJB1EYAeBw2Sd8kkkm/zgqlvFQu7aCh4nzFum+W6vj+t5vD5nXatvqD2IrEYGiqz1ojb9Hs6ylt6CyxJE598rkvU0rmao+0nr8ykyCu+zyDQDlESoj5VXkBkIU4VLneVXp6kpNxlJiHEX3YWpyNQZ4bLgMIujZaLdsQDreAofm5RPN+h31/wAACAASURBVEag1p69LsN3kdZyACa0cxB36t/iW3+oxaJoxzadSzNiFE4I33eEn1lgemMR8dUVAq+RNXqUBetaXz2nygjOTCELmYSP4DmyfmYY/B1kbS14Q2SR0LdvbOKhMyOWpJmzLludOrc2GeH9CtZosIpLcso7J84iDx0WqbNTQlm+6KSU+jWWrCFnHS+Xe3ozKWvT0dMErd0pqXG18TFR+UAEIgKujYdv0XXA481Zvg3bKZnaWoywC8b1T2vppFMaikabVtZ6rBAdRKCBwAtkrXND48waog427jxuavz4aGDdzQsITQ+VhwwaCjspfFOYekA55c6FJzLdM1bJ4IxE7D/CZ+fJq3U707J2rwACl0V/0qPP3q3yKHWWXx0NcjpG1rpNALuCTDeWdeqAueF3Kaghe6kbyVqQ4O8zCNTas1jWdsCsTrApn1zHoTZJz8W6kawjFHQcR+A5si6sOs6HhEGAppFLXVwDHcPBKHqYFMQypgvfJb7+bSjsEqnopH9xW4sx5XS0zGq8U8naK4xx9iMKWNhmLYT0Hsk1r5MTWiJWKCDpnIh1ftyy1m0g5ZnK866ETxZHy1u78wzCM8m6Agy9mwgkks2j+W/I6DGnv8od35RPIR3JOgeXzy8i8DRZ2yHUTMkaa8nWCMTnSSMsXgpfgve3i8ZsSjdOHueV3WItNTy76SyIRSaZBGWPzWEyl4zV3Wke18875cmQXMgEv0JY+N2UifrEOSkpOKVPPsF1GllrbD1Zo1vk6m80j61Rkk3FCiiBo7KwnZo8j20bkaSCr7dkdOdAjXaQrAUu/r4ZgUSyecYHSdclQ3uXzunBdA3iz2vCZyIgCHyGrF3j1EOhYV91bLSZQs8WKKUVmNttUknhyyvpX+QrQ7f2Q0K6yEGoh16t7bJIpJOXIc9u1TvYML6HLvvTZO3fL1nRiaxBht5S1otjpK4Ke7xHAAUEjEUw6BzBT89rS0r51WTtiN/loffOizKTFP5XcOSctcWFT59B4DmyPr7AzPaLkx6Jb1XRFzGcDiJQQOB9ZL347U1Y7GQaKxpmaWEShtSjNaoIw1mCgWBhgT78di3D3+pFksJXnsEphCMhjkikcpl1iw83EZ3/MOXZl7GsM1Z1u73VaQ7cbeuofHypbiBOZZ2bBWbw1x0ZqW363RvmdUQaF8dIOk3W3k86PX59QIiHeflA0FF56feBO4ZjBGG7d0u26pmtYzPm+MJahLR8VirnfhM+Vv6u8yPlqPJNYvWwh4+KSicRiAjE9h59xOE7vqIqxBe/8g250TUcmDSVtm7pFCRrjQbdryPwHFm7gzP0fLNWslgcVt5fbaq3zG5VNPZL65wwt+pPC8NwaHluCErckF51SHpe3VB3vr930fVD50LVAHm7g7tRWxDztAotCan4i0rEYvTxYLm7ehe+bEln3v+FhzYZ+UV52+K3ZO2KBv7xPdUQnusYYasKOleqw2XIcqvEcFAMsMKCttJUAIhcwjTcqAuUpcyL26mFtK/eLDarYNfGp5KI3kSACBCBjhB4kqw7erMfquprZFTo0W8wWQJxh1ED6Z2EeI5M3ciG6tToPMJBD8ZS1+HKDas79hFi/qpToOIapxuBySpmIvAEswwOPhIBIvCDCJCsOxDqa2TdwYu9qYrE501AMhsiQAQuiwDJ+rKiSRWDkPhPDNgG2AbYBu7dBh7//v1LzEDX5RDAB8q/OgLEp44NQ4gAEfgNBKDnSNYXlyXJqC0g4tPGh6FEgAj0jwDJugMZkozaQiI+bXwYSgSIQP8IkKw7kCHJqC0k4tPGh6FEgAj0jwDJugMZkozaQiI+bXwYegQBnCEgpx1m8XHWgDr7wR22lN/WFQ948mnNAUxZdnwkAq8gQLJ+BbUPpzmfjPpWVOfj82GBs7gvIPDcNyCnG/qK4pwCf9e1nEhAsv6CCH+8SJJ1BwI+n4z6VlTn49NBI2EV/4jAX74BFI306ZAfkvUfxcHkGwRI1htIrudxPhn1rajOx+d6bYI1ejcCfX8D70aD+V0PAZL19WSyqdH5ZNS3ojofn41I6PFzCPzlG/BX0+o7DWhZ/1wD+foLkay/LoL9CpxPRn0rqvPx2ZcRY/SOwHPfgL91y1844y7ywTWyCgKStQKDzrcgQLJ+C4znZnI+GfWtqM7H51z5MvcrIPDcNxAXmOGKWbVSXN6EZC1I8PddCJCs34XkifmcT0Z9K6rz8TlRuMz6MghU77cubN2KZB2uetXPeCGS9WXE+jMVIVl3IMpPkFHPiuoT+HTQTFjFPyJQI9jcP39e3R3w46ovct3E+WPdmJwIkKw7aAOfIKOacsn98+crKKpP4NNBM2EV/4qAI91hHec0+4x72IdH2pKFIjbfQPB7jImuS3H+Wj2mvzcCJOsO5P8RMupYUX0Enw7aCav4BgSWaR0HWTj2WB/DuCrudgWUiXheR0XqiGMXoT1WTeZvqCmzuBkCJOsOBP4xMupUUX0Mnw7aCqtIBIjAbyJAsu5AriSjtpCITxsfhhIBItA/AiTrDmRIMmoLifi08WEoESAC/SNAsu5AhhAS/4kB2wDbANvAvdvA49+/fx1Q1n2riA+Uf3UEiE8dG4YQASLwGwhAz5GsLy5LklFbQMSnjQ9DiQAR6B8BknUHMiQZtYVEfNr4MJQIEIH+ESBZdyBDklFbSMSnjQ9DiQAR6B8BknUHMiQZtYVEfNr43Df0+Jn352OEQ1PskaSbMrMzyDfh9Lg1AiTrDsRPMmoLifi08blv6JXI+oAUSNYHQLpvFJJ1B7InGbWFRHza+Nw3lGR9X9n/3puTrDuQKcmoLSTi08bnvqFHyXpZ5zGd5T2M86qu8jBh9qxw5I9LPmZ/nnjhXuuEfT4MbstEvtM8Fe/GTnnQdWcESNYdSJ9k1BYS8Wnjc9/Qg2Q9j+sjEq0nUblACyQ+TIm8F0eoMvfsyXoYt5d9bDG3ZI0raXGxR+wUuIt0cHHIlPy2mdDnxgiQrDsQPsmoLSTi08bnvqFPkLWwswELFvOWPNPd7z7/YYqUa1LbB03W2p1iuZu6CuWlGHTdGQGSdQfSJxm1hUR82vjcN/QgWa8+HoaiR2VF+7va1XWZj+T2BI109q7rOtaKoGsLyWr+9UwZciMESNYdCJtk1BYS8Wnjc+fQZAVnKBSJcVnnaVyHYVidoV2Mo/MhWWs06D4XAZL1ufi+JXeSURtG4tPG586hGFouDVPX/B1WmMN2bK2s4SKIL5L1Ws6Xw+BFkOkZECBZd9AUSEZtIRGfNj63DoV1/IClnOaVl3l0fjLVjEVkozysWGD2iAS/XQiGeWy7wCwmbQJtCTrP1y1cwzA756ybKN45kGTdgfRJRm0hEZ82PrcPXSa/tUrmnEG2ibvXFZbuoOajR72orLDFKqZ91bKGRAr5cuvW7ZtqCwCSdQudi4SRjNqCID5tfBj6IQSWeZ3GcR0L/8es7w/Vk8V0iQDJugOxkYzaQiI+bXwYSgSIQP8IkKw7kCHJqC0k4tPGh6FEgAj0jwDJugMZkozaQgI+/CcGbANsA7/eBh7//v1ra0OGfhUBNED+1REgPnVsGEIEiMBvIAA9R7K+uCxJRm0BEZ82PgwlAkSgfwRI1h3I8KNk5E5tyre2nAeS2+M6F/I3lysUwpXXR/FR5dL5owh8+BvwKGL72NGjS38Ud75WEwGSdROeawQeIyP/seOoxPb/uO5tI8EBDdh3aq8KrGCBAyZ2y/R1Kt2VALJO9Qn7VkHe8RSpSrnK+xg+KgGdP4rAN74BlCmHpARYC99EOnSlDL07vaz0gZSj0/eGCJCsOxD6MTKC0tCHOXjCy49atOTYevllnWd1hV8tKhRTYttaLHcncCmarY96B5J1FUsG1BBQ7UeiFNqnbXMSsfT7xDegCTtvu4U6mNLcKWs4lKXc0Z5KI08mAz7cAQGSdQdS/gRZL+ECg7aVnFkQwA6KqaJkdF6Px8NfjiB4BwXmFCfKBpMHpeXShdOmJI+WZXIMHymYv7+LwN/I+uVvAOeRTWNq3zk5589aAG7IHaNLiztQJZ2s5keZ9DGpOhnd90OAZN2BzI+R0V8VVbhpqIpHIX/EFUW0zN4SX5Z1WbxFMs+TU0Ag3FzpYNgPo37eyvF5z8HPVSG3Tqr1Wt22rUYwg26DQKGNSvtUGNQsa2mTKmrmLOSfxcAj8jcj2oU6+GSeoOOI04L8cU45pqG230yhKHrdCAGSdQfCPkzWj4edP3bWqR1agxUclYN695cVVVREQtCetHHPLwgaxB2PUlblicKU3wXnNz+U5U6yVmjReQwBXJbxwW8AbV/Wawg7O79sOipYzwPWgZQ+vvByfq1I+F7HA1NQx0BhrB9BgGTdgSAPk/Uf5qzfQdbTmDoGGPZ+iCJz1oKmbFxi4BVaJGsMI2pFRrLuoGVerYoFyzd2JlNdpc0lH+96+RsAGTuyDkPZecbF52Vd5tndnw0rWkaf0leC8GmdpmmdKx3eYrb0/FkESNYdiPZTZN2ee65c36eUoVaC2h2HygtYm3grlF0gfDNn3V7BfgyfQuH0+jEE/k7WL30Dkaz9EHi0tlVnFX66L+pv3ZrWacblH6mTW0wbvoWHHnn6McnxdfYRIFnvY/T1GMfI6ANDgCVlEckaC2KSNW0sawxNWk0VMbVkHb3dXPhDhhaVd8l5DJ9SSvr9FgLf+wZMW1Xk7fDNnzPQ7TeA70hNB7nrO7Nh9Sw9H++BAMm6AzkfIiMohD8Mg89jtlr7IC61oUOrgOqZVeNxGLwOGkPKCHzpGyiNHOl2rd2littwknUJI/r5hbQ8bvTiLeEQWZfILVq96QWtYhB/WCS6Ny/++7+SH371EJ6xrGU4MMxT61wlvfZz7tL7bCJ5j0P4VNLS+4cQKLWZD3wDkaxntX3LTemM6yTbEhswo6Os13fYb6cy/dTIj0G/iQAt6w7keoSMipbxQUUF69gvkHkWjETyOenKs1skg1UzJasnbHMpjpCXFG+lekfwqSSl9w8h8J1vIGxfdB1SPVyNRZShA7uzslu+FS8KWtY/1CTf+iok67fCeU5mu2QEIixZxkfIGnHMHNnxd3AkH5jWK5zFr24dkWfaQvPA8PwuWfs9ptE6NwvMoPTqlv8uPsdfiTF7ReBL3wDgks6uW8mNMwbcAUPjOskJJ2H/NNowFpSlFd8ebFrWvTa6z9abZP1ZvF8qrU1Gvge/uxYLlioWeumh6KBEipbtbk2hlKZVTkIU68BtRzFbTcLCM5xyVihI0u0W14jQxqeRkEE/gsC3voEMPtfxbRxmsszrNJXIOl8prvOd11F/szqI7lshQLLuQNwko7aQiE8bH4YSASLQPwIk6w5kSDJqC4n4tPFhKBEgAv0jQLLuQIYko7aQgA//iQHbANvAr7cBbt1qc8HXQ9EA+VdHgPjUsWEIESACv4EA9BzJ+uKyJBm1BUR82vgwlAgQgf4RIFl3IEOSUVtIxKeND0OfRMBtMxxX2Xl1KDV2Vqj91NjlsLtD41DGjEQEPAIk6w5aAsmoLSTi08bnPqHZXn05OW/z274YBnj56yqx1XG71aqEJ/Za2/PB/TnlbcLGlrNxHQ/96wNXSjWg368jQLLuQMIko7aQiE8bn/uE/u3WrS1O/o72/BCTUjzc3745RkAOamlkALKWswqWSXUizIFGhffaVoI+P44AyboDAZOM2kIiPm187hNaIDVDeh6J2kE8IMt4gt7GGpez7xO5Rlxx4FA4pU/uppZ83EFEJi+bHnUZxLJGPOV+DGJ1o2xa1hHvmzpI1h0InmTUFhLxaeNzn9C/kvXePHMhf1zY4a6GfROZVo7lvY8M+aY1BEjWNWQu5E8yaguD+LTxuU/oB+6zzixcP1c9rqPyX5Z8nluOQ7Xj4c6SF0tafodhTRa1WNb+d5hkwPw+EuWbJgRI1gmLy7pIRm3REJ82PvcJLVi+Tw2DP2lZz3KBji13e/sXOhGFOe2CYLiKvAAKvRwCJOsOGgLJqC0k4tPG5z6hljTdez9J1g9cOGPmmPVz7W7prFyZwxbgd4a2cUuXrAgfUL5Y2fE3t9QlY/7eCQGSdQfSJhm1hUR82vjcJ/QDw+Clq2jXjKzxrOPt3M1eW/Dm5ZbnfR9p8k0tAiRri8cln0hGbbEQnzY+twktWbBPWNbb4eujyG0JVeeFee3WfmvE3VrTYm1zJfhRKfx6PJJ1BxImGbWFRHza+NwmtGTBHibrzBp+CrQtWa+xLlgt3p6vpmX9FNi3jUyy7kD0JKO2kIhPG5+7hGprNr7zQbKG9YvTyl77K5D1uqwLFn+XrP2sEJJ1BggfiwiQrIuwXMuTZNSWB/Fp43OLUJCinieWlz5C1ogz2MNKJPmxX0/WWCi2WZz2eKz5wSijO+psXqdw6Mkw7AyDPwa/AG2Y4mlnx+rFWL+EAMm6A2mSjNpCIj5tfH4/VPYx77wphqYfIEZ1gAku4NgZpt7JdV03C8z2UzAGEXgWAZL1s4h9IT7JqA068Wnjw1AiQAT6R4Bk3YEMSUZtIRGfNj4MJQJEoH8EHFn/z//97woH/4kB2wDbANsA2wDbwDXbwANkzT8i0CsCUCz8Ow8B4nsetsiZ+J6L76/kjnZCsv4Vad70PajszhU88SW+5yLA3I8gQLI+ghLjXBoBksm54iG+xPdcBJj7EQRI1kdQYpxLI/BbZOLvR375fI4TJHU1fP21lH84wKS0H/sE3I5meTV82/We12lUF4u4rW/jOtvbP7MslnWeZ/8/TW6rHA6gwWEwm33pw7i6behZDu3HvVPi2uGoW7P67cI/Fkqy/hjULOgsBC6l7JagjEo3NxUZeFmn7J7i/EQrOV3rWwrlUvi6Q8HaZ22321nptLF2irNDr4Zv833jMaop1uL2r7cIG5iP6wRSdMe6pbT5HvW87ePYVkvoJTIHGbcOtamTNcrD/eHtzoau7/fcJOvvYc+S34TA95QdTqFS1gHI2J2k9cgUDOI81keFrJ2FoawVo7CcslKHeLwJs2ey+R6+5Vo6y7p4lWVJked5kKxzRI4/B9Kbtx3S/JQ2nAiXLGSNeTiGVQrF96K+C9P2EcecQFcj5Zp/LKRwPrs/SAdlf6sTLLU7+kuyPooU410Wge+RybJOchqWKB35zdGq+Yd4mrCjwkKapsWQF3LO8/fwLb8PyFrp93Ik5wv5qM6U61htj//0ltv3OkRXw7cKaMGqrsY1AZqsvZUdJzEMGYObs0tPTHiNlGv+UgmEq3zDqXVD6k1IxEv/kqwvLR5W7ggC31N27yNrvKcMEUaFtWRWyBEwTojzPXzLL3OcrEvpNXGUwj/vdzV8ywj4u8L16JDrYOqRJXFvelIB80CS4zj6USaZ744FZkQO/wJZT4Uz2GHZP1z52qKXjIWsl9Wd3/5QxC1ROvglWXcgJFaxjcD3lF2FrHH+tCiu+JsPg0Mx5Vaff06KJw9vzcu1MfpL6PfwlVpnWLnLMTJsNgQhafPfQBy59xefv4/v/svLndvNIWtkUxpBgl84kz3ODYe1HWLcIv8HLiwRD6lSgayjVS5x1ics62XuYn46vppykKwVGHT2icD3lB2UhBCzJWNv/WE2LCiSAxNjSCNDc3+zHt8rx+/hK++xQ7AVgih3hgIpjKO/yUr/mnlWKfv83+/j237HZQ7TMRnOjmA3HU77HUjOS2Rp8Tn46xavpY7ZI6zkR50SrxfIOlvoWe8Ap7yvMOXUQoVk3UKHYV0g8D1ll1vWU5ojjcNyg7oisTRElyAmWScsrOsFsrYZqKedvFTMTzm/136ffMOMrJ9KDQt5Q+yaKIM7HyGpWdYg8UHWGRTI2lQO4WnoO04zmTjXfyBZX19GrOEOAt9TdhlZD+HeYVhrUEzBaoOSwjyd+2/cSQxLRXQVLWstdD9fWlf2ZWvO5bBZFLUla427LvVT7u+13yffUMg6s1prcjFD2oZ0Ue5WDsUhdJNOkzLc8r1o/9I7WbL228GE6Evxr+lHsr6mXFirJxD4nrLLyDowbbSQg3Jb8Lu7lxOElOakSdZPNIBWVKPsEXFLEt+2tL7XflvAFcIUWeu560LMbGGYXyiGOelE7KVV+YVOl5FfRsrzGOa4M/9NhTKydtVJU06b6Bf1IFlfVDCs1nEEvqfsoAT0nPXkT2VyVvS0zpjrwz5O2ZcKZVXZ1wly1ittSdZK/tLpWbE3NlhEWEksbucvVpZKB6dR9s5jHePwqY9Lss4wqz1Gsp7XaQr7lEtD25AL5pT1PPUBOTxnWetKPknW2GXh1pJgC+CBxSS6qC+6SdZfBJ9FvweBr5K1EEZUZF4ROGWF05Hc3PW0+hWsOHaxoBygyLLtJCWy/hapfA9ftA/fIfKDFtZCAh4ybeAPo0kjE7FlHSCJb+EqdfwuvlKLA7/SxgtRS+3VRDsghxpZa4tcFpiZvGURp/VUT9JuwtatOMolB6NMXawQJ1krkdLZJwLfU3bLilWumn4X7AEd8fEvaRvLgvOUC1b14pUFlFFa2eplsFV+GCbfxvuExL6HLyDcjjjIinlYzXo0AnPPMUyAcQuR9MrvwXWM0nCsf87xl+Sf+P0mvk+9X5Gsfevfttcs54oc4loONxplZe1yMCQvpKvydmcR2CkkFRqcvsPnt4ZtTyzDcanuhMFI4tscruBDsr6CFFiHPyFwRWXnSAZ7SxssAIvOE3vh9aEYZT9x+DULdgpJzvL6Gr7h0AwD4aKHV7ORCix8ys5ZX9EhcgodB8yUD5mhZX2w5aBNZlMIrsMU2mcOvcnVkK4JSQ+lzsDOvmi3fay0PzvlGtYp7J//vTm23OTx/QeS9fdlwBr8EYGvkckf691LcuJ7rqSI77n4/kruJOtfkeSN3wONmP/EgG2AbeDX28Dj379/N1b1fPXeEcAHyr/zECC+52GLnInvufj+Su5oJyTrX5HmTd+Dyu5cwRNf4nsuAsz9CAL3JmssepCtNxott3LxtRNusLCotahIF/OK22xX0Rn8oc46mx7dJJNzpUZ8ie+5CDD3IwiQrM1S0wBZbeWiW+KvT+Hxbuz1xJYdbLA/soXBbBspHSoQ/OIeUiVJu3I1bGXAJl6QdSmBSvurTpLJuZIlvsT37whg+2J5P7PbFVHSw4cLLWzpCmmdPv5T3ocrcXrEe5I19r3icApNyvq8W9kyUyPNwhYGIdFDZH2g8Uh+eQuw/uroRJJ1DhWf34QAyfpNQFay6QNf6JqtoVI2PEoX1tT2QiPfwmE2gtXGQCrlDbKu5eH3WMfRzk1+6Z0OqGWp1Vd+b0nWcahak7WGv+YvcTbEiIYYLj/PiT5vAUhrzshNjUU3fJx8ZQzlUCdH1jh4A/mGvbgunStXjr4s3Asrdf/B3z6UXb/AE99zZdcdvqX90Hs6s2DgAFVt3Czz9sASY1BVTyprkbUrZZ2am8Bht33nwKFnWtYtyVo20sdDJ3Arkr7RfKfhxfToaeKsZzVPrRtfURCSNzb7o3GGAxvgxlnS0+ivksvPrJV8faPyFvU8qXpvOhDF0n/Ssztl15kUiO+5AusO31fJ2ihZx9T2kBXoxsKhK9EqfpasKx2EkjRJ1iVUvu6nemFCnK7dqPtWM+sYlms6PUoN2whBgnDDewmpVl8zlonTlzxh4xf3rbo5b5WXzkMak/ziJid9SxPnrDVadL8Tge7I5J0v/4G8usP3KbKGbvNXxqYRRQxle53nBx7ldDncYped+hf1JQThdfeEkcVsSN6fwY9RSjVMruuppzlj2hRX9OoHxP1yEWgn99q6JQQLyExDKGGYSFhCcyt6mCY7l7Mh+mwuJZaJhpmGwFNj836pc4CS021D0qjcgjY9xK7fSyp7k9/ulF1nciG+5wqsO3w1CQo0Ua+Jh/1NRkwwlpzVG3RduOsdQ9UYXXRGiFg/Jl9laJnsK/66ntod0oouxaN2m6wv9HA/stbgoyGMfuhZ855ESQ0s+EDgj3GdwvVvGA63Izsyd93Y9qUan24g2t3qRJh46GkK4Zs569RjlHf55d/ulF1nwiC+5wrs+vjmi8sKd1EXjJSkU0GmMifsp/CEi3eRhRGi1vjIrVvuVruYCcl6F8euI8iNRxg2kZXhcXgkWLyxAQbyk0Pl3RAOGqw0QI8EiDQOz6SWamGKZI0GFhalDXKVYio3zdPkyW2ZMZSWdYSCjvcicH0yee/7fjq37vAt6Zqo10roKYJWFq7Rl1H31kYika8iZdQhzm8rf128Ksssxg1laf1tjSCdyXXcaCf3GgZ3Q8rD+gBJg1w1qS52NeLGstZyyxqsGx4Pi80w95x6kjqRXf2oQ442lmq8rD467193d6fsOhMI8T1XYL3h63Sd1puAp0XWSjfptFtdViBdk68Oh1tGNrW/klVO1nYY1Ax9b+ui8rmI84ZkjQ5amIs2DWFd/b2mqWdXJ2sMdycL1zXA0MuLadyQeYoj8pZGgd9oheeWtfQyC6erSXrJL/6qDyL63cTRm7LrTSzE91yJ9Ybvdvpvh6yDgYTrS7URs9VlBdI1OjoLn8ew8DfzF3GRrAWJH/g1DSG8DxY4YFgcnK63RsnrhiFwb5Fj4Res9DRHbdIIYcdtYelggLyhynOci0Fala8UL/HkOf6SrCMUdLwXgd7I5L1vf35uXeHrdFoyaCI6JV0aA70DuhEjmqIOt7qsQLom30K4y7rin5M17pcXQ8gZSMmY2tYlq/wFHtFObjYMrlA3DUH5BycEaEZOXEO1WwvcPmmV1JC18ocTYTLs7hvHss44ptTtrU4NyZH/Lllniz7MAjNY7emjyKrxc49dKbsO0Se+5wqtH3xBijL0nGGyo0vXFfoK64Ow7Wpc/TKhRJY+twLpwggpLDCzpRfSIYIma5tg80Sy3kByMY9KA3M9QNcLm2Iv8GjN62QNYk75SePAqT2zDMu7QvwH4RpoPi/UyRaDo1i9K14/yu5dxPNmmAAABQhJREFUb/zZfIjvuXj3gS9GER/1+wcqutQht3jDIho+YW2Qy09ZurB6ZbV3RNzkC92YETx0J/J/FIwTknWEkQ4icAkE+lB2l4DqpUoQ35dgO5yoB3xxrkNpWi6+pCHV6Avz1i3kjUStgsRgSV4FC1l24KRIxuUI/6EPrVLBR8gacWCYFdYHqZwu4UQ7ue8w+CVEwEr8FYEelN1f3/Gb6YnvuegT33Px/ZXcSda/IskbvwcaMf+JAdsA28CvtwFa1jcmul94dXyg/DsPAeJ7HrbImfiei++v5I52QrL+FWne9D2o7M4VPPElvuciwNyPIECyPoIS41waAZLJueIhvsT3XASY+xEESNZHUGKcSyNAMjlXPMSX+HoEwl7peIGGwgX7oQuHOKkYmbOwDSuLcfyxnpfbSlvYAns87+vEJFlfRxasyYsIkExeBO5gMuJ7EKgXo3WDb+uUxOrWrRoohW1apajI1+zFLt0o2MoLYeogq01+6djnq3M6ybrUQOjXFQLdKLuuUE2VJb4JizNcveCb9kUv6xyOZI54VMlaXeNrSDe7adCEqQNOTL41Uq75S+2WFXdlt/7Su7VifTeMZP1d/Fn6GxDoRdm94VW/kgXxPRf2LvDVB4yEIW8clRyt3nidsFiqJQtY47hHsCHuX8gadT44NE+y1rKhmwichEAXyu6kd/9EtsT3XJSvjy+IFcd8+qM9N0d+Ah5DqiW8thb2AwRvLOrsKNFNvp7gJ91JCOlTXqqToDsYy7SOWVk4o1yGvknWJZnRjwi8GYHrK7s3v/CHsyO+5wJ+fXxBtP7CoQm3DJbPDo2XFJXRyi3p/Bl8f4ystwPa27xcHTKyHrJ66/K0u1z/7/uinXCf9fflwBr8AYHrK7s/vNwFkhLfc4XQD74ZKcKaFms1GwYfxWSN0GVpcWZ4djNgkTArt27Fq4Rd/tu8vPe0RoLWxB3qpMvT7ljlizlI1hcTCKvzPAL9KLvn3+0KKYjvuVLoBV99xe/ziIBQ7bB3Grr289y4aXDD8WZ4XZGy2Sqm/HXFNEHDzfusNTp0E4HPI9CLsvs8Mu8pkfi+B8daLj3gu4A03e1WGA4HuU7r4u6olgVl+jfNBRffGcTpLHKfVzY6bZPUyNpZ5nK39kGyzgrS1rR22wpc5wnthMPg15EHa/ICAj0ouxde6zJJiO+5ougC33lax3FyW7aWRU5FKZNkm/j83dZYrOaHwfE7rBmPJsCrZO0XtfnhdskrJXOu3LLOCtH11O4sl8s8kqwvIwpW5FUEulB2r77cBdIR33OF0CO+nrDLJFklvkWIGnjqtJ6wh2lepRsQEW+RdYyk84qe65qTNYfBFTh0EoEvINCjsvsCTC8XSXxfhu5Qwm7wXeYVq8Hd3K87ZKRMkluyXlbZk53mpPO0i1sNjq1c47wk0q4sMLPA5nmFUE3WNsHmaVvnTZSve6CdcBj862JgBf6CQDfK7i8v+cW0xPdc8C+PL0gv7Ik2RBrmjfO90nahmCdhrMq2VnOZYBd0CBKjZ/u3kSZbhBb2fo8PdeqZiItkLUjwlwhcA4HLK7trwPRyLYjvy9AdStgDvmmeWr/SvJZI8piVWiZrnbtzL/Oan2yq48zjY0XnYLtVDCPtauuWTqTdoSMyjFgwd+0/tBNa1teWEWu3g0APym7nFS4dTHzPFQ/xPRffX8k9kjUc/CcGbANsA2wDbANsA9dsA/8PSOhw9eJmM/cAAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.埋点收集数据 （10分）\n",
    "在广告位置 曝光和点击收集\n",
    "![image-2.png](attachment:image-2.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.确定H0,H1 （10分）\n",
    "A组为实验组 B组为对照组\n",
    "\n",
    "一类指标:\n",
    "实验组人均停留时长不明显小于对照组。\n",
    "\n",
    "$$\n",
    "H_1:\\mu_B - \\mu_A < a_1\n",
    "$$\n",
    "$$\n",
    "\\mu_B = \\mu_A - a_1\n",
    "$$\n",
    "$$\n",
    "\\bar x_A - \\bar x_B\n",
    "$$\n",
    "\n",
    "\n",
    "$$\n",
    "n_A = kn_B\\ and\\ n_B = (1 + \\frac{1}{k})(\\sigma\\frac{z_{1- \\alpha} \\ \\ \\ +  z_{1-\\beta}}{\\mu_A - \\mu_B} )^2\n",
    "$$\n",
    "$$ \\sigma = \\sqrt {\\frac{\\sigma_A^2}{n_A}+\\frac{\\sigma_B^2}{n_B}} $$\n",
    "\n",
    "\n",
    "H0: control_stime - treatment_stime >= 2 * std(control_stime) \n",
    "\n",
    "H1: control_stime - treatment_stime < 2 *std(control_stime)  \n",
    "      \n",
    "      \n",
    "二类指标:\n",
    "实验组广告点击率显著大于对照组\n",
    "$$\n",
    "H_1:\\pi_A - \\pi_B > a_1\n",
    "$$\n",
    "\n",
    "\n",
    "H0: treatment_p- control_p <=0\n",
    "\n",
    "H1: treatment_p- control_p >0 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5.确定显著水平α （5分)\n",
    "\n",
    "一类错误使用默认值 α= 0.05  \n",
    "二类错误使用默认值 β= 0.2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha =0.05\n",
    "beta =0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6.计算样本量 （15分）\n",
    "一类指标没日志数据，我们不计算，只计算二类指标\n",
    "\n",
    "从二类指标的H0和H1 中我们知道时单侧(右侧) 检验 ，从统计量的确定中，我们得知是比值，所以统计量的计算公式如下\n",
    "$$\n",
    "n_A = kn_B\\ and\\ n_B = (\\frac{\\pi_A(1-\\pi_A)}{k} + \\pi_B(1-\\pi_B))(\\frac{z_{1- \\alpha} +  z_{1-\\beta}}{\\pi_A - \\pi_B} )^2\n",
    "$$\n",
    "\n",
    "其中 treatment_p 为 $p_A$,   control_p为$p_B$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1203863045004612"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算treatment_p\n",
    "# 计算对照组广告的点击率（由于converted值为0，1 所以求占比和均值结果一致）\n",
    "control_p = data.converted[(data.group==\"control\") & (data.landing_page==\"old_page\")].mean()\n",
    "control_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10589344218918344"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算p0*(1-p0) = \n",
    "control_p_1_control_p = control_p * (1-control_p)\n",
    "control_p_1_control_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.6448536269514729"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算z 1-alpha\n",
    "# 均值之差在大样本下其分布为近似正态分布\n",
    "Z1_alpha = stats.norm.isf(alpha,loc=0,scale=1)\n",
    "Z1_alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8416212335729142"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算z 1-beta\n",
    "# 二类错误的概率\n",
    "Z1_beta = stats.norm.isf(beta,loc=0,scale=1)\n",
    "Z1_beta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.13538630450046119 0.014999999999999986\n"
     ]
    }
   ],
   "source": [
    "#H1为 treatment_p- control_p >0，为计算样本量，这里需要指定一个非0的值， 此值的绝对值越小样本量越大， \n",
    "# treatment_p- control_p >0.015\n",
    "#我们先指定一个偏小的值 0.015\n",
    "treatment_p = control_p + 0.015\n",
    "p_p0 = treatment_p - control_p\n",
    "print(treatment_p,p_p0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.11705685305416959"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# p * (1-p)\n",
    "p_1_p = treatment_p * (1-treatment_p)\n",
    "p_1_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6126.235378834391"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n = (control_p_1_control_p + p_1_p) * ((Z1_alpha +Z1_beta) / p_p0 )**2\n",
    "n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>date</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-02</td>\n",
       "      <td>2859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-03</td>\n",
       "      <td>6590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-04</td>\n",
       "      <td>6578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-05</td>\n",
       "      <td>6427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-06</td>\n",
       "      <td>6606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-07</td>\n",
       "      <td>6604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-08</td>\n",
       "      <td>6687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-09</td>\n",
       "      <td>6628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-10</td>\n",
       "      <td>6654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>6688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-12</td>\n",
       "      <td>6522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-13</td>\n",
       "      <td>6552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-14</td>\n",
       "      <td>6548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-15</td>\n",
       "      <td>6714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-16</td>\n",
       "      <td>6591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-17</td>\n",
       "      <td>6617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-18</td>\n",
       "      <td>6482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-19</td>\n",
       "      <td>6578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-20</td>\n",
       "      <td>6534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-21</td>\n",
       "      <td>6749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-22</td>\n",
       "      <td>6596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-23</td>\n",
       "      <td>6716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-24</td>\n",
       "      <td>3754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-02</td>\n",
       "      <td>2853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-03</td>\n",
       "      <td>6618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-04</td>\n",
       "      <td>6541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-05</td>\n",
       "      <td>6505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-06</td>\n",
       "      <td>6747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-07</td>\n",
       "      <td>6609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-08</td>\n",
       "      <td>6700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-09</td>\n",
       "      <td>6615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-10</td>\n",
       "      <td>6696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>6673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-12</td>\n",
       "      <td>6637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-13</td>\n",
       "      <td>6508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-14</td>\n",
       "      <td>6600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-15</td>\n",
       "      <td>6549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-16</td>\n",
       "      <td>6545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-17</td>\n",
       "      <td>6538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-18</td>\n",
       "      <td>6603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-19</td>\n",
       "      <td>6552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-20</td>\n",
       "      <td>6679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-21</td>\n",
       "      <td>6560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-22</td>\n",
       "      <td>6669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-23</td>\n",
       "      <td>6633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-24</td>\n",
       "      <td>3681</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        group landing_page        date  user_id\n",
       "23    control     old_page  2017-01-02     2859\n",
       "24    control     old_page  2017-01-03     6590\n",
       "25    control     old_page  2017-01-04     6578\n",
       "26    control     old_page  2017-01-05     6427\n",
       "27    control     old_page  2017-01-06     6606\n",
       "28    control     old_page  2017-01-07     6604\n",
       "29    control     old_page  2017-01-08     6687\n",
       "30    control     old_page  2017-01-09     6628\n",
       "31    control     old_page  2017-01-10     6654\n",
       "32    control     old_page  2017-01-11     6688\n",
       "33    control     old_page  2017-01-12     6522\n",
       "34    control     old_page  2017-01-13     6552\n",
       "35    control     old_page  2017-01-14     6548\n",
       "36    control     old_page  2017-01-15     6714\n",
       "37    control     old_page  2017-01-16     6591\n",
       "38    control     old_page  2017-01-17     6617\n",
       "39    control     old_page  2017-01-18     6482\n",
       "40    control     old_page  2017-01-19     6578\n",
       "41    control     old_page  2017-01-20     6534\n",
       "42    control     old_page  2017-01-21     6749\n",
       "43    control     old_page  2017-01-22     6596\n",
       "44    control     old_page  2017-01-23     6716\n",
       "45    control     old_page  2017-01-24     3754\n",
       "46  treatment     new_page  2017-01-02     2853\n",
       "47  treatment     new_page  2017-01-03     6618\n",
       "48  treatment     new_page  2017-01-04     6541\n",
       "49  treatment     new_page  2017-01-05     6505\n",
       "50  treatment     new_page  2017-01-06     6747\n",
       "51  treatment     new_page  2017-01-07     6609\n",
       "52  treatment     new_page  2017-01-08     6700\n",
       "53  treatment     new_page  2017-01-09     6615\n",
       "54  treatment     new_page  2017-01-10     6696\n",
       "55  treatment     new_page  2017-01-11     6673\n",
       "56  treatment     new_page  2017-01-12     6637\n",
       "57  treatment     new_page  2017-01-13     6508\n",
       "58  treatment     new_page  2017-01-14     6600\n",
       "59  treatment     new_page  2017-01-15     6549\n",
       "60  treatment     new_page  2017-01-16     6545\n",
       "61  treatment     new_page  2017-01-17     6538\n",
       "62  treatment     new_page  2017-01-18     6603\n",
       "63  treatment     new_page  2017-01-19     6552\n",
       "64  treatment     new_page  2017-01-20     6679\n",
       "65  treatment     new_page  2017-01-21     6560\n",
       "66  treatment     new_page  2017-01-22     6669\n",
       "67  treatment     new_page  2017-01-23     6633\n",
       "68  treatment     new_page  2017-01-24     3681"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#查看我们的样本量是否满足最低样本量 \n",
    "dd=data.groupby([\"group\",\"landing_page\",\"date\"],as_index=False)[\"user_id\"].count()\n",
    "dd[((dd.group ==\"control\") & (dd.landing_page==\"old_page\"))|((dd.group ==\"treatment\") & (dd.landing_page==\"new_page\"))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "group      landing_page\n",
       "control    new_page          1928\n",
       "           old_page        145274\n",
       "treatment  new_page        145311\n",
       "           old_page          1965\n",
       "Name: user_id, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#查看我们的样本量是否满足最低样本量 \n",
    "data.groupby([\"group\",\"landing_page\"])[\"user_id\"].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7 利用统计工具实现检验(45分) \n",
    "\n",
    "1. 读取ab_test.csv 数据 \n",
    "\n",
    "2. 计算统计量  \n",
    "\n",
    "3. 计算统计量的显著性P值 \n",
    "\n",
    "4. 用统计量的显著性P值与显著性α比较做决策 \n",
    "\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0.106383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0.113809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0.113781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0.097826</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       group landing_page  converted\n",
       "0    control     new_page   0.106383\n",
       "1    control     old_page   0.113809\n",
       "2  treatment     new_page   0.113781\n",
       "3  treatment     old_page   0.097826"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = data[data.date=='2017-01-03'].groupby([\"group\",\"landing_page\"],as_index=False)[\"converted\"].mean()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 我们只需要控制组的旧页面和对照组的新页面比例即可\n",
    "statistic_t = df.converted[2] - df.converted[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "求$$ \\sigma = \\sqrt {\\frac{p_A (1-p_A)}{n_A}+\\frac{p_B (1-p_B)}{n_B}} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6590 6618\n"
     ]
    }
   ],
   "source": [
    "# 用size求个数，为了求方差\n",
    "n1 = data[data.date=='2017-01-03'].converted[(data.group==\"control\") & (data.landing_page==\"old_page\")].size\n",
    "n2 = data[data.date=='2017-01-03'].converted[(data.group==\"treatment\") & (data.landing_page==\"new_page\")].size\n",
    "print(n1,n2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.005526379176809786"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigma  = np.sqrt(df.converted[2] * (1-df.converted[2])/n2  + df.converted[1] * (1-df.converted[1])/n1 )\n",
    "sigma"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "H0决定拒绝域，<=则说明拒绝域在右侧，所以，使用stats.norm.cdf，我们只能求出左侧累计概率，需要使用1-\n",
    "\n",
    "H0: treatment_p- control_p <=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5020359187180234"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "statistic_p = 1-stats.norm.cdf(statistic_t,0,sigma)\n",
    "statistic_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "显著性P > α 实验组点击率 <= 对照组\n"
     ]
    }
   ],
   "source": [
    "if(statistic_p > alpha):\n",
    "    print(\"显著性P > α 实验组点击率 <= 对照组\")\n",
    "else:\n",
    "    print(\"显著性P < α 实验组点击率 > 对照组\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#封装如下\n",
    "def ABtest_P(df: pd.DataFrame, group_col: str = None, value_col: str = None ,alpha:float = 0.05):\n",
    "    '''\n",
    "    :param df: 被分析DateFrame对象\n",
    "    :param alpha: 显著性\n",
    "    :param group_col: 组列的名字，默认为df的第一列\n",
    "    :param value_col: 值列的名字,默认为df的第2列\n",
    "    :return: tongjiliang p_value p_type\n",
    "    '''\n",
    "        # 列名\n",
    "    if not group_col:\n",
    "        group_col = df.columns[0]\n",
    "    if not value_col:\n",
    "        value_col = df.columns[1]\n",
    "        \n",
    "    temp = df.groupby(group_col,as_index=False)[value_col].mean()\n",
    "    temp_n = df.groupby(group_col,as_index=False)[value_col].count()\n",
    "    tongjiliang =temp.iloc[0,1] - temp.iloc[1,1]\n",
    "        \n",
    "    diff_error =np.sqrt(temp.iloc[0,1] *(1-temp.iloc[0,1])/temp_n.iloc[0,1] +  temp.iloc[1,1] *(1-temp.iloc[1,1])/temp_n.iloc[1,1])\n",
    "    \n",
    "    tongjiliang_left_p = stats.norm.cdf(tongjiliang,0,diff_error)\n",
    "    tongjiliang_right_p = 1-stats.norm.cdf(tongjiliang,0,diff_error)\n",
    "    tongjiliang_site_p = tongjiliang_left_p *2;\n",
    "    if tongjiliang_site_p >1:\n",
    "        tongjiliang_site_p = tongjiliang_right_p*2\n",
    "    \n",
    "#     在封包的时候，我们左右双侧检验都用上，提高通用性\n",
    "    temp_l = [[temp.iloc[0,0],temp.iloc[1,0],tongjiliang,\"左侧\",tongjiliang_left_p,np.where(tongjiliang_left_p<alpha,\"显著\",\"不显著\")],\n",
    "              [temp.iloc[0,0],temp.iloc[1,0],tongjiliang,\"右侧\",tongjiliang_right_p,np.where(tongjiliang_right_p<alpha,\"显著\",\"不显著\")],\n",
    "              [temp.iloc[0,0],temp.iloc[1,0],tongjiliang,\"双侧\",tongjiliang_site_p,np.where(tongjiliang_site_p<alpha,\"显著\",\"不显著\")]]\n",
    "    temp = pd.DataFrame(temp_l,columns =[\"p\",\"p0\",\"统计量\",\"检测\",\"p_value\",\"结果\"])\n",
    "    \n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>converted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>treatment</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>treatment</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>control</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>treatment</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>control</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        group  converted\n",
       "38  treatment          0\n",
       "41  treatment          0\n",
       "57    control          0\n",
       "72  treatment          0\n",
       "84    control          0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp  = data[data.date=='2017-01-03'].loc[((data.group==\"control\") & (data.landing_page==\"old_page\")) \n",
    "                  | ((data.group==\"treatment\") & (data.landing_page==\"new_page\")),[\"group\",\"converted\"]]\n",
    "temp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>p0</th>\n",
       "      <th>统计量</th>\n",
       "      <th>检测</th>\n",
       "      <th>p_value</th>\n",
       "      <th>结果</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>control</td>\n",
       "      <td>treatment</td>\n",
       "      <td>0.000028</td>\n",
       "      <td>左侧</td>\n",
       "      <td>0.502036</td>\n",
       "      <td>不显著</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>control</td>\n",
       "      <td>treatment</td>\n",
       "      <td>0.000028</td>\n",
       "      <td>右侧</td>\n",
       "      <td>0.497964</td>\n",
       "      <td>不显著</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>control</td>\n",
       "      <td>treatment</td>\n",
       "      <td>0.000028</td>\n",
       "      <td>双侧</td>\n",
       "      <td>0.995928</td>\n",
       "      <td>不显著</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         p         p0       统计量  检测   p_value   结果\n",
       "0  control  treatment  0.000028  左侧  0.502036  不显著\n",
       "1  control  treatment  0.000028  右侧  0.497964  不显著\n",
       "2  control  treatment  0.000028  双侧  0.995928  不显著"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ABtest_P(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
